var source = require('./source/testSource.js');
var async = require('async');
//var readability = require('node-readability-cheerio');
var mysqlDao = require('./dao/mysqlDao.js');
var cheerio = require('cheerio');
var request = require('request');
var schedule = require('node-schedule');
function todo() {
    console.log('start');
    async.mapLimit(source, 1, function (item, cb) {
            console.log('start:' + 'http://etfdb.com/etf/' + item.symbol);
            request({
                timeout: 20000,
                url: 'http://etfdb.com/etf/' + item.symbol,
                headers: {
                    Host: 'etfdb.com',
                    'Upgrade-Insecure-Requests': 1,
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
                }
                // headers: {
                //     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
                //     'Accept-Encoding': 'gzip, deflate, sdch',
                //     'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
                //     'Cache-Control': 'max-age=0',
                //     'Connection': 'keep-alive',
                //     'Cookie': '__qca=P0-1789510694-1481083140057; __gads=ID=6a0a0b579eaa49ad:T=1481083137:S=ALNI_MbA8eWJeMZCe6UgX-st1qSToz_15Q; optimizelyEndUserId=oeu1481083141536r0.9660946579977514; broker=0; bSe=1481087899623; invstr_seg=0; iSe=1481087899629; __asc=83c41beb158d79c55748d945931; __auc=c08c06f7158d76f36ac5f6207c2; __utmt=1; optimizelySegments=%7B%22174920844%22%3A%22gc%22%2C%22174987565%22%3A%22false%22%2C%22175033320%22%3A%22direct%22%7D; optimizelyBuckets=%7B%7D; popup_domination_hide_lightbox2=true; __utma=148430563.1589503510.1481083141.1481083141.1481086097.2; __utmb=148430563.13.9.1481086941855; __utmc=148430563; __utmz=148430563.1481083141.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); optimizelyPendingLogEvents=%5B%5D',
                //     'Host': 'etfdb.com',
                //     'Upgrade-Insecure-Requests': '1',
                //     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36'
                // }
            }, function (err, res, body) {
                if (err) {
                    console.error(err + ' in doTT.js');
                    cb(null, item.symbol);
                    return;
                }
                console.log('http://etfdb.com/etf/' + item.symbol);
                var $ = cheerio.load(body);
                //readability.read('http://etfdb.com/etf/' + item.symbol, function (err, $) {
                //    if (err) {
                //        console.log('error:::::http://etfdb.com/etf/' + item.symbol);
                //        console.error(err);
                //        return;
                //    }
                //    console.log('http://etfdb.com/etf/' + item.symbol);
                var liArr = $('.list-unstyled').eq(0).find('li');
                var topArr = $('#holdings-collapse .holdings-ol').find('li');
                var topArrL = topArr.length;
                var top_ten = [];
                var sector = [];
                for (var i = 0; i < topArrL; i++) {
                    var arrr = $('#holdings-collapse .holdings-ol').find('li').eq(i).text().trim().split('\n');
                    if (arrr.length <= 3) {
                        top_ten.push({
                            cpy_name: arrr[0],
                            cpy_code: '',
                            percent: arrr[2]
                        });
                    } else {
                        top_ten.push({
                            cpy_name: arrr[arrr.length - 1],
                            cpy_code: arrr[0],
                            percent: arrr[2]
                        });
                    }
                }
                var chartArr = $('#holdings-collapse .chart');
                var sectorArr = [];
                if (chartArr.length > 0) {
                    sectorArr = JSON.parse($('#holdings-collapse .chart').eq(1).attr('data-chart-series'));
                }
                //console.log(sectorArr);
                var sectorArrL = sectorArr.length;
                for (i = 0; i < sectorArrL; i++) {
                    var sectorTxt = sectorArr[i][0].split('(');
                    sector.push({
                        sec_name: sectorTxt[0].trim(),
                        percent: sectorTxt[1].substr(0, sectorTxt[1].length - 1),
                        val: sectorArr[i][1]
                    });
                }
                var market_cap = [];
                var market_capArr = [];
                if (chartArr.length > 0) {
                    market_capArr = JSON.parse($('#holdings-collapse .chart').eq(2).attr('data-chart-series'));
                }
                var market_capArrL = market_capArr.length;
                for (i = 0; i < market_capArrL; i++) {
                    var market_capTxt = market_capArr[i][0].split('(');
                    market_cap.push({
                        mkt_name: market_capTxt[0].trim(),
                        percent: market_capTxt[1].substr(0, market_capTxt[1].length - 1),
                        val: market_capArr[i][1]
                    });
                }
                var returnData = [];
                var returnArr = $('#performance-collapse .col-md-6');
                var returnArrL = returnArr.length;
                for (i = 0; i < returnArrL; i++) {
                    var returnSingle = returnArr.eq(i);
                    var val = returnSingle.find('.relative-metric-bubble').text().trim() == 'No Ranking Available' ? '' : returnSingle.find('.relative-metric-bubble').text().trim();
                    returnData.push({
                        name: returnSingle.find('.relative-metric-name').text().trim(),
                        value: val
                    });
                }
                var index = liArr.length > 5 ? liArr.eq(6).find('.pull-right').text() : liArr.eq(4).find('.pull-right').text();
                var ratio = liArr.length > 5 ? liArr.eq(2).find('.pull-right').text() : liArr.eq(1).find('.pull-right').text();
                var issue_date = liArr.length > 5 ? liArr.eq(4).find('.pull-right').text() : liArr.eq(3).find('.pull-right').text()
                var obj = {
                    name_en: $('.data-title').text().trim().split('\n')[1],
                    issuer: liArr.eq(0).find('.pull-right').text(),
                    issue_date: issue_date,
                    time: $('#barchart-stock-quote .bcSimpleQuoteTime .bcSimpleQuoteTimeValue ').text(),
                    index: index,
                    ratio: ratio,
                    introduction: $('#analyst-collapse p').text().trim(),
                    aum: $('.list-unstyled').eq(2).find('li').eq(0).find('.pull-right').text(),
                    '1m_vol': $('.list-unstyled').eq(4).find('li').eq(0).find('.pull-right').text(),
                    '3m_vol': $('.list-unstyled').eq(4).find('li').eq(1).find('.pull-right').text(),
                    pe_ratio: $('#valuation-collapse .relative-metric-bubble-data').text(),
                    yield: $('#dividend-collapse .relative-metric-bubble-data').text(),
                    num_holding: $('#holdings-collapse .relative-metric-bubble-data').eq(0).text(),
                    top_ten: top_ten,
                    sector: sector,
                    market_cap: market_cap,
                    returnData: returnData
                };
                var tttArr = obj.time.split('/');
                obj.time = '20' + tttArr[2] + '-' + tttArr[0] + '-' + tttArr[1];
                var ob = Object.assign({}, item, obj);
                cb(null, ob);
                //});
            });
        },
        function (err, data) {
            if (err) {
                console.log(err);
            }
            //console.log(data);
            mysqlDao.todo(data);
        }
    );
}
// todo();
// console.log('doTest.js 线程');
// var rule = new schedule.RecurrenceRule();
// var times = [];
// var l = 24;
// for (var i = 0; i < l; i += 6) {
//     times.push(i);
// }
// rule.hour = times;
// rule.minute = 0;
// var j = schedule.scheduleJob(rule, function () {
//     console.log("执行任务" + (new Date()).toString());
//     todo();
// });
var rule = new schedule.RecurrenceRule();
var times = [];
var l = 60;
for (var i = 0; i < l; i++) {
    times.push(i);
}
rule.minute = times;
var j = schedule.scheduleJob(rule, function () {
    console.log('doTest.js 开始' + new Date().toString());
    var date = new Date().getTime();
    var now = new Date(date);
    var hour = Number(now.getHours());
    var minute = Number(now.getMinutes());
    if (hour != 5 && minute != 30) {
        console.log('不在轮训时间，北京时间->' + hour + ':' + minute);
        return;
    }
    console.log("执行任务" + (new Date()).toString());
    todo();
});
